# coding=utf-8
import requests,re,sys,time,sys,threading
import Queue
from optparse import OptionParser
sj = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())+'.html'
reload(sys)
sys.setdefaultencoding('utf-8')
lock = threading.Lock()
threads = []
q = Queue.Queue()
with open('test.txt','r') as fo:
	for li in fo.readlines():
		ur3 = li.strip('\n')
		q.put(ur3)
def CMS():
	while not q.empty():
		try:
			ur3 = q.get(True, 1)
		except:
			#ur3.task_done()
			break
		try:
			# ur1 = re.search(r"(http)", ur3, re.I)  # 匹配网站是否有http，没有则添加头
			# if ur1 == None:
			# 	ur2 = 'http://' + ur3
			# else:
			# 	ur2 = ur3
			zt = requests.get(url=ur3,allow_redirects=False,verify=False,timeout=10,)#网站返回状态
			de = zt.status_code
			print ur3
			code = [200,401,403]
			if de in code:
				head = zt.headers
				s = 'Server' in head.keys()
				if s == True:
					serve = head['Server']
					print serve#服务器类型
				else:
					serve ='None'
				x = 'X-Powered-By' in head.keys()
				if x == True:
					by = head['X-Powered-By']
					print by#语言程序
				else:
					by = 'None'
					print by
				nr = zt.text
				tit = re.findall(r'<title>(.*?)</title>',nr,re.S)
				#print tit
				if tit:
					tit = tit[0]
					print tit
				else:
					tit = 'None'
					print tit
				p = re.findall(r':(\d+)',ur3,re.I)
				if p: #端口
					p = p[0]
					print p
				else:
					p = '80'
					print p
				aa = '<a target=_blank href='+ur3+'>'+ur3+"  ||   "+str(de)+"   ||  "+serve+"    ||   "+by+"   ||   "+tit+'</a><br>'
				print aa
				#jg_url = open('20170420.html',"ab+")
				jg_url = open('%s' % sj,"ab+")
				jg_url.write(aa+'\n')
				jg_url.close()
				print '\033[35;1msuss....\033[0m'+'\n'
		#lock.release()
			#db = MySQLdb.connect("localhost","root","321abcD","tb",charset="utf8")
			#cursor = db.cursor()
			#print str(ur2),str(p),str(serve),str(by),str(head),str(tit[0])
			#sql="INSERT INTO `tb`.`cms` (`URL`,`port`,`Server`,`X-Powered-By`,`head`,`title`) VALUES ('%s','%s','%s','%s','%s','%s')" % (str(ur2),p,str(serve),str(by),str(he),tit)
			#cursor.execute(sql)
			#db.commit()
			#print '\033[35;1msuss....\033[0m'+'\n'
			#db.close()
			#cursor.close()
		except Exception,e:
			print e
def main():
	parser = OptionParser('./xxx.py t num')
	parser.add_option('-t',type='int',dest='thread',default='20',help='the is threading')
	(options,args) = parser.parse_args()
	thre = options.thread
	for i in range(thre):
		t = threading.Thread(target=CMS)
		threads.append(t)
		t.start()
	for t in threads:
		t.join()
if __name__ == '__main__':
	main()